import numpy as np
import pandas as pd
from sklearn.metrics import accuracy_score, classification_report
import re
import json
import statsmodels.api as sm
from scipy.stats import mode
import matplotlib.pyplot as plt
from data_utils.preprocess import process


def labels_count():
    """
    统计标签数量
    :return:
    """
    path = r'../data/eval/virus_eval.txt'
    with open(path, 'r', encoding='utf-8') as f:
        data = json.load(f)
    print(len(data))
    labels = [data['label'] for data in data]
    # print(data)
    # print(set(labels))
    df = pd.DataFrame(labels, columns=['Labels'])
    print(df['Labels'].value_counts())


def length_count():
    """
    统计长度
    :return:
    """
    paths = [
        r'../raw/train/usual_train.txt',
        r'../raw/train/virus_train.txt',
        r'../raw/eval/usual_eval.txt',
        r'../raw/eval/virus_eval.txt'
    ]
    for p in paths:
        with open(p, 'r', encoding='utf-8') as f:
            data = json.load(f)
        data = [d['content'] for d in data]

        # process
        data, _ = process(data, [0]*len(data), drop_empty=False)

        lengths = []
        for d in data:
            lengths.append(len(d))

        # 直方图
        print('均值', np.mean(lengths),
              '标准差', np.std(lengths),
              '众数', mode(lengths)[0][0],
              '中位数', np.median(lengths),
              '最大值', np.max(lengths),
              '最小值', np.min(lengths))
        plt.style.use('seaborn-white')
        plt.hist(lengths, bins=1000, histtype='step')
        plt.show()

        # 累积分布函数
        ecdf = sm.distributions.ECDF(lengths)
        x = np.linspace(min(lengths), max(lengths))  # x轴数据上值对应的累计密度概率
        y = ecdf(x)
        plt.step(x, y)
        plt.show()


if __name__ == '__main__':
    # labels_count()
    length_count()
    pass
